package com.czk.java;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.Arrays;
import java.util.List;

/**
 * @Author:ChenZhangKun
 * @Date: 2021/12/14 9:47
 */
public class JavaSparkDemo01 {
    public static void main(String[] args) {
        SparkConf sparkConf = new SparkConf().setAppName("java").setMaster("local[*]");
        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

        jsc.setLogLevel("WARN");
        // 加载文件
        JavaRDD<String> fileRDD = jsc.textFile("F:\\IdeaProjects\\bigData\\spark\\datas\\test.txt");
        JavaRDD<String> wordRDD = fileRDD.flatMap(line ->
             Arrays.asList(line.split(" ")).iterator()
        );
        // 计算数据
        JavaPairRDD<String, Integer> wordCountRDD = wordRDD.mapToPair(word -> new Tuple2<>(word, 1));

        JavaPairRDD<String, Integer> reduceByKey = wordCountRDD.reduceByKey((a, b) -> a + b);

        List<Tuple2<String, Integer>> collect = reduceByKey.collect();

        collect.forEach(System.out::println);
    }
}
